In [1]:
import os
import numpy as np
import pandas as pd
from glob import glob
import shutil

# image
import cv2
from skimage.io import imread

# TensorFlow
import tensorflow as tf
from tensorflow.keras import layers, models

# Visualisation libraries

## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output

## progressbar
from tqdm import tqdm

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse, Polygon
from matplotlib.font_manager import FontProperties
import matplotlib.colors as mcolors
from matplotlib.colors import LinearSegmentedColormap
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from matplotlib import cm
# plt.style.use('seaborn-whitegrid')
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")
Home / Image Processing
Natural Scenes Image Classification

Dataset Description¶

Content¶

This Data contains around 25k images of size 150x150 distributed under 6 categories. {'buildings' -> 0, 'forest' -> 1, 'glacier' -> 2, 'mountain' -> 3, 'sea' -> 4, 'street' -> 5 }

The Train, Test and Prediction data is separated in each zip files. There are around 14k images in Train, 3k in Test and 7k in Prediction.

In [2]:
def Header(Text, L = 100, C = 'Blue', T = 'White'):
    BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN, 'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
         'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(BACK[C] + FORE[T] + Style.NORMAL + Text + Style.RESET_ALL + ' ' + FORE[C] +
          Style.NORMAL +  (L- len(Text) - 1)*'=' + Style.RESET_ALL)
    
def Line(L=100, C = 'Blue'):
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(FORE[C] + Style.NORMAL + L*'=' + Style.RESET_ALL)
    
Start_Path = 'natural_scenes'
Target = 'Class'
Data = pd.DataFrame({'Path': glob(os.path.join(Start_Path, '*', '*', '*', '*.jpg'))})
Data['File'] = Data['Path'].map(lambda x: x.split('\\')[-1])
Data['Class'] = Data['Path'].map(lambda x: x.split('\\')[-2].title())
Data['Dataset'] = Data['Path'].map(lambda x: x.split('\\')[-3].title().replace('Seg_',''))
#
Header('A Sample of the Dataframe')
Data = Data.reindex(sorted(Data.columns), axis=1)
display(Data.sample(10))
display(pd.DataFrame({'Number of Instances':[Data.shape[0]], 'Number of Attributes':[Data.shape[1]]}).style.hide_index())

def Path_Tree(startpath, Extension, sep = ' ' * 3):
    Folders_with_Images = []
    C = ['Red', 'Green', 'Magenta', 'Cyan']*len(os.listdir(startpath))
    BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN,
            'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
            'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
    for root, _, files in os.walk(startpath):
        level = root.replace(startpath, '').count(os.sep)
        if level >0:
            indent = sep* (level)+ '└──'
            print(indent +  BACK[C[level]] + Fore.BLACK + Style.NORMAL + os.path.basename(root) + Style.RESET_ALL)
        else:
            title = os.path.basename(root)
            print(Style.RESET_ALL + Fore.BLUE + Style.NORMAL + '=' * (len(title) +1) + Style.RESET_ALL)
            print(Back.BLACK + Fore.CYAN + Style.NORMAL + title + Style.RESET_ALL)
            print(Style.RESET_ALL + Fore.BLUE + Style.NORMAL + '=' * (len(title) +1)+ Style.RESET_ALL)
        subindent = ' ' * 4 * (level + 1)
        for file in files[:1]:
            if file.endswith(Extension):
                Folders_with_Images.append(root)
                List = os.listdir(root)
                print(level* sep, Fore.BLUE + Style.NORMAL + 
                      '%i %s files: ' % (len(List), List[0].split('.')[-1].upper()) + Style.RESET_ALL +
                      '%s'%', '.join(List[:5]) + ', ...')
    return Folders_with_Images
_ = Folders_with_Images = Path_Tree(Start_Path, '.jpg')
A Sample of the Dataframe ==========================================================================
Class Dataset File Path
16034 Street Train 2111.jpg natural_scenes\seg_train\seg_train\street\2111...
3086 Buildings Train 10744.jpg natural_scenes\seg_train\seg_train\buildings\1...
16419 Street Train 5253.jpg natural_scenes\seg_train\seg_train\street\5253...
14292 Sea Train 6962.jpg natural_scenes\seg_train\seg_train\sea\6962.jpg
6940 Forest Train 5899.jpg natural_scenes\seg_train\seg_train\forest\5899...
3570 Buildings Train 14568.jpg natural_scenes\seg_train\seg_train\buildings\1...
11421 Mountain Train 3227.jpg natural_scenes\seg_train\seg_train\mountain\32...
4290 Buildings Train 2285.jpg natural_scenes\seg_train\seg_train\buildings\2...
2817 Street Test 22677.jpg natural_scenes\seg_test\seg_test\street\22677.jpg
10570 Mountain Train 15269.jpg natural_scenes\seg_train\seg_train\mountain\15...
Number of Instances Number of Attributes
17034 4
===============
natural_scenes
===============
   └──seg_pred
      └──seg_pred
       7301 JPG files: 10004.jpg, 10005.jpg, 10012.jpg, 10013.jpg, 10017.jpg, ...
   └──seg_test
      └──seg_test
         └──buildings
          437 JPG files: 20057.jpg, 20060.jpg, 20061.jpg, 20064.jpg, 20073.jpg, ...
         └──forest
          474 JPG files: 20056.jpg, 20062.jpg, 20082.jpg, 20089.jpg, 20091.jpg, ...
         └──glacier
          553 JPG files: 20059.jpg, 20087.jpg, 20092.jpg, 20109.jpg, 20111.jpg, ...
         └──mountain
          525 JPG files: 20058.jpg, 20068.jpg, 20071.jpg, 20085.jpg, 20093.jpg, ...
         └──sea
          510 JPG files: 20072.jpg, 20076.jpg, 20077.jpg, 20081.jpg, 20099.jpg, ...
         └──street
          501 JPG files: 20066.jpg, 20067.jpg, 20069.jpg, 20070.jpg, 20075.jpg, ...
   └──seg_train
      └──seg_train
         └──buildings
          2191 JPG files: 0.jpg, 10006.jpg, 1001.jpg, 10014.jpg, 10018.jpg, ...
         └──forest
          2271 JPG files: 10007.jpg, 10010.jpg, 10020.jpg, 10030.jpg, 10037.jpg, ...
         └──glacier
          2404 JPG files: 10.jpg, 100.jpg, 10003.jpg, 10009.jpg, 10011.jpg, ...
         └──mountain
          2512 JPG files: 10000.jpg, 10001.jpg, 10002.jpg, 10008.jpg, 10023.jpg, ...
         └──sea
          2274 JPG files: 1.jpg, 10016.jpg, 10041.jpg, 10053.jpg, 10061.jpg, ...
         └──street
          2382 JPG files: 1000.jpg, 10015.jpg, 10019.jpg, 10022.jpg, 10036.jpg, ...

Dataset Distribution¶

In [3]:
def DatasetDist(Inp, Target, PD):
    Table = Inp[Target].value_counts().to_frame('Count').reset_index(drop = False).rename(columns = {'index':Target})
    Table = Table.sort_values(by = [Target])
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()), 2)
    
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=PD['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])

    # Right
    fig.add_trace(go.Pie(labels=Table[Target].values, values=Table['Count'].values,
                         pull=PD['pull'], textfont=dict(size= PD['textfont']),
                         marker=dict(colors = [Colors_dict[x] for x in Table[Target].unique()],
                                     line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=PD['hole'])
    fig.update_layout(height = PD['height'], legend=dict(orientation=PD['legend_orientation']),
                      legend_title_text= PD['legend_title'])
    
    # Left
    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%%%.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)

    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= PD['TableColors'][0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = PD['tablecolumnwidth'],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [PD['TableColors'][1], PD['TableColors'][1]]),
                                  align=['center','center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + PD['Title'] + '<b>', 'x':PD['title_x'],
                                     'y':PD['title_y'], 'xanchor': 'center', 'yanchor': 'top'})
    fig.show()
    
Colors_dict = dict(zip(Data[Target].unique().tolist(),
                       ['DarkRed', 'DarkGreen', 'Silver', 'Sienna', 'SteelBlue','Gray']))
Pull = [0 for x in range((len(Data[Target].unique().tolist())-1))]
Pull.append(.05)
PD = dict(PieColors = Colors_dict, TableColors = ['DarkGreen','GhostWhite'], hole = .4,
          column_widths=[0.5, 0.5],textfont = 14, height = 400, tablecolumnwidth = [.1, .05, .08], pull = Pull,
          legend_title = Target, legend_orientation = 'v',
          Title ='Train Set', title_x = 0.5, title_y = 0.84)
del Pull
DatasetDist(Data.loc[Data['Dataset'] == 'Train'], Target = Target, PD = PD)
PD.update(dict(TableColors = ['Indigo','GhostWhite'], Title ='Test Set'))
DatasetDist(Data.loc[Data['Dataset'] == 'Test'], Target = Target, PD = PD)
In [4]:
fig, ax = plt.subplots(4, 5 , figsize = (12, 12))
_ = fig.suptitle('A Sample of Train Dataset', fontweight='bold', fontsize = 18)
ax = ax.ravel()
for i, row in Data.loc[Data['Dataset'] == 'Train'].sample(len(ax)).reset_index(drop =True).iterrows():
    _ = ax[i].imshow(imread(row['Path']))
    _ = ax[i].set_title('%s' % row[Target], fontweight='bold', fontsize = 12, color = Colors_dict [row[Target]])
    _ = ax[i].axis("off")
    _ = ax[i].set_aspect(1)
fig.tight_layout()
In [5]:
batch_size = 128
(Img_Height, Img_Width, _) = imread(Data['Path'][0]).shape

Header('Train Images')
train_ds = tf.keras.preprocessing.image_dataset_from_directory(directory= 'natural_scenes\\seg_train\\seg_train',
                                                               shuffle=True,
                                                               seed=123,
                                                               image_size=(Img_Height, Img_Width),
                                                               batch_size=batch_size)
Header('Validation Images Data Generator', C = 'Green')
val_ds = tf.keras.preprocessing.image_dataset_from_directory(directory= 'natural_scenes\\seg_test\\seg_test',
                                                             image_size=(Img_Height, Img_Width),
                                                             batch_size=batch_size)
Line()
Train Images =======================================================================================
Found 14034 files belonging to 6 classes.
Validation Images Data Generator ===================================================================
Found 3000 files belonging to 6 classes.
====================================================================================================
Modeling: Keras Multi-layer Perceptron (MLP) for Image Classifications

A multi-layer perceptron (MLP) is a class of feedforward artificial neural network (ANN). The algorithm at each iteration uses the SparseCategoricalCrossentropy to measure the loss, and then the gradient and the model update is calculated. At the end of this iterative process, we would reach a better level of agreement between test and predicted sets since the error would be lower from that of the first step.

In [6]:
num_classes = len(Data[Target].unique())
data_augmentation = tf.keras.Sequential([layers.experimental.preprocessing.RandomFlip("horizontal", 
                                                                                   input_shape=(Img_Height, Img_Width, 3)),
                                      layers.experimental.preprocessing.RandomRotation(0.1),
                                      layers.experimental.preprocessing.RandomZoom(0.1)])

model = models.Sequential(name = 'Multi_Class_MLP',
                          layers = [data_augmentation,
                                    layers.experimental.preprocessing.Rescaling(1./255, input_shape=(Img_Height, Img_Width, 3)),
                                    layers.Conv2D(16, 3, padding='same', activation='relu'),
                                    layers.MaxPooling2D(),
                                    layers.Conv2D(32, 3, padding='same', activation='relu'),
                                    layers.MaxPooling2D(),
                                    layers.Conv2D(64, 3, padding='same', activation='relu'),
                                    layers.MaxPooling2D(),
                                    layers.Flatten(),
                                    layers.Dense(128, activation='relu'),
                                    layers.Dense(num_classes)])
model.summary()
tf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True,
                          show_layer_names=True, expand_nested = False, rankdir= 'LR')
Model: "Multi_Class_MLP"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
sequential (Sequential)      (None, 150, 150, 3)       0         
_________________________________________________________________
rescaling (Rescaling)        (None, 150, 150, 3)       0         
_________________________________________________________________
conv2d (Conv2D)              (None, 150, 150, 16)      448       
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 75, 75, 16)        0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 75, 75, 32)        4640      
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 37, 37, 32)        0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 37, 37, 64)        18496     
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 18, 18, 64)        0         
_________________________________________________________________
flatten (Flatten)            (None, 20736)             0         
_________________________________________________________________
dense (Dense)                (None, 128)               2654336   
_________________________________________________________________
dense_1 (Dense)              (None, 6)                 774       
=================================================================
Total params: 2,678,694
Trainable params: 2,678,694
Non-trainable params: 0
_________________________________________________________________
Out[6]:

Compiling and fitting the model

In [7]:
# Number of iterations
IT = 21

model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])

# Training the model
history = model.fit(train_ds, validation_data=val_ds, epochs=IT, verbose = 0)
In [8]:
def Search_List(Key, List): return [s for s in List if Key in s]

Metrics_Names = {'loss':'Loss', 'accuracy':'Accuracy', 'mae':'MAE', 'mse':'MSE', 'recall': 'Recall'}

def Table_modify(df, Metrics_Names = Metrics_Names):
    df = df.rename(columns = Metrics_Names)
    df = df.reindex(sorted(df.columns), axis=1)
    df.insert(loc = 0, column = 'Iteration', value = np.arange(0, df.shape[0]), allow_duplicates=False)
    return df

Validation_Table = Search_List('val_',history.history.keys()) 
Train_Table = list(set( history.history.keys()) - set(Validation_Table))
Validation_Table = pd.DataFrame(np.array([history.history[x] for x in Validation_Table]).T, columns = Validation_Table)
Train_Table = pd.DataFrame(np.array([history.history[x] for x in Train_Table]).T, columns = Train_Table)
Validation_Table.columns = [x.replace('val_','') for x in Validation_Table.columns]

Train_Table = Table_modify(Train_Table)
Validation_Table = Table_modify(Validation_Table)

# Train Set Score
score = model.evaluate(train_ds, batch_size = batch_size, verbose = 0)
score = pd.DataFrame(score, index = model.metrics_names).T
score.index = ['Train Set Score']
# Validation Set Score
Temp = model.evaluate(val_ds, batch_size = batch_size, verbose = 0) 
Temp = pd.DataFrame(Temp, index = model.metrics_names).T
Temp.index = ['Validation Set Score']
score = score.append(Temp)
score.rename(columns= Metrics_Names, inplace = True)
score = score.reindex(sorted(score.columns), axis=1)
display(score.style.set_precision(4))
Accuracy Loss
Train Set Score 0.8710 0.3594
Validation Set Score 0.8200 0.5582
In [9]:
def Plot_history(history, PD, Title = False, metrics_names = [x.title() for x in model.metrics_names]):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    Colors = ['OrangeRed', 'MidnightBlue', 'purple']
    for j in range(len(metrics_names)):
        fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history[metrics_names[j]].values,
                                 line=dict(color=Colors[j], width= 1.5), name = metrics_names[j]), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, PD['yLim']], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    if not PD['Table_Rows'] == None:
        ind = np.linspace(0, history.shape[0], PD['Table_Rows'], endpoint = False).round(0).astype(int)
        ind = np.append(ind, history.index[-1])
        history = history[history.index.isin(ind)]
    T = history.copy()
    T[metrics_names] = T[metrics_names].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    TableColors = PD['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color=TableColors[0],
                   fill_color=TableColors[0], align=['center','center'], font=dict(color=TableColors[1], size=12), height=25),
                   columnwidth = PD['tablecolumnwidth'], cells=dict(values=Temp, line_color=TableColors[0],
                      fill=dict(color=[TableColors[1], TableColors[1]]),
                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    if Title != False:
        fig.update_layout(plot_bgcolor= 'white',
                      title={'text': Title, 'x':0.46, 'y':0.94, 'xanchor': 'center', 'yanchor': 'top'},
                          yaxis_title='Frequency')
    fig.show()

PD = dict(Table_Rows = 25, yLim = 2, tablecolumnwidth = [0.3, 0.4, 0.4], TableColors = ['DarkSlateGray','White'])
Plot_history(Train_Table, Title = 'Train Set', PD = PD)
Plot_history(Validation_Table, Title = 'Validation Set', PD = PD)
In [10]:
val_images = []
val_labels = []
for x, y in list(val_ds):
    val_images.extend(x)
    val_labels.extend(y)
Pred = model.predict(val_ds)
class_names = val_ds.class_names
class_names = [x.title() for x in class_names]
Prob = np.array(tf.nn.softmax(Pred[0]))
for i in tqdm(range(len(Pred[1:]))):
    Temp = np.array( tf.nn.softmax(Pred[i]))
    Prob = np.column_stack((Prob, Temp))

Pred = pd.DataFrame(data = Prob.T, columns = class_names)
Pred['Actual Label'] = [class_names[x] for x in val_labels]
100%|███████████████████████████████████████████████████████████████████████████| 2999/2999 [00:00<00:00, 27011.85it/s]
In [11]:
display(Pred.round(2))
Buildings Forest Glacier Mountain Sea Street Actual Label
0 0.03 0.00 0.00 0.00 0.00 0.97 Glacier
1 0.03 0.00 0.00 0.00 0.00 0.97 Street
2 1.00 0.00 0.00 0.00 0.00 0.00 Sea
3 0.02 0.00 0.05 0.87 0.06 0.00 Street
4 0.02 0.16 0.00 0.00 0.00 0.82 Mountain
... ... ... ... ... ... ... ...
2995 0.37 0.02 0.00 0.60 0.00 0.01 Sea
2996 0.28 0.00 0.04 0.01 0.66 0.01 Sea
2997 0.99 0.00 0.00 0.00 0.00 0.01 Sea
2998 0.31 0.00 0.11 0.45 0.05 0.08 Sea
2999 0.00 0.00 0.81 0.14 0.04 0.00 Sea

3000 rows × 7 columns


References¶

  1. Kaggle Dataset: Intel Image Classification
  2. Tensorflow API Documentation